package com.pengheng.sink;

import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.common.serialization.SimpleStringEncoder;
import org.apache.flink.api.common.typeinfo.Types;
import org.apache.flink.api.connector.source.util.ratelimit.RateLimiterStrategy;
import org.apache.flink.configuration.MemorySize;
import org.apache.flink.connector.datagen.source.DataGeneratorSource;
import org.apache.flink.connector.datagen.source.GeneratorFunction;
import org.apache.flink.connector.file.sink.FileSink;
import org.apache.flink.core.fs.Path;
import org.apache.flink.streaming.api.CheckpointingMode;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.sink.filesystem.OutputFileConfig;
import org.apache.flink.streaming.api.functions.sink.filesystem.bucketassigners.DateTimeBucketAssigner;
import org.apache.flink.streaming.api.functions.sink.filesystem.rollingpolicies.DefaultRollingPolicy;

import java.time.Duration;
import java.time.ZoneId;

public class SinkFile {
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        // 每个目录中，都有并行度个数的文件在写入
        env.setParallelism(2);
        // 必须开启checkpoint，否则一直都是.inprogress
        env.enableCheckpointing(2000, CheckpointingMode.EXACTLY_ONCE);
        GeneratorFunction<Long, String> generatorFunction = new GeneratorFunction<Long, String>() {
            @Override
            public String map(Long value) throws Exception {
                return "Number:" + value;
            }
        };
        DataGeneratorSource<String> dataGeneratorSource = new DataGeneratorSource<>(generatorFunction, Long.MAX_VALUE, RateLimiterStrategy.perSecond(1000), Types.STRING);
        DataStreamSource<String> dataGen = env.fromSource(dataGeneratorSource, WatermarkStrategy.noWatermarks(), "data-generator");

        DateTimeBucketAssigner<String> bucketAssigner = new DateTimeBucketAssigner<>("yyyy-MM-dd HH", ZoneId.systemDefault());
        //设置输出文件前缀后缀信息
        OutputFileConfig outputFileConfig = OutputFileConfig.builder()
                .withPartPrefix("atguigu-")
                .withPartSuffix(".log")
                .build();
        //设置滚动策略，1分钟，或者最大为1M的文件
        DefaultRollingPolicy<String, String> rollingPolicy = DefaultRollingPolicy
                .builder()
                .withRolloverInterval(Duration.ofMinutes(1))
                .withMaxPartSize(new MemorySize(1024))
                .build();
        //输出到文件系统
        FileSink<String> fileSink = FileSink.<String>forRowFormat(new Path("f:/tmp"),
                        new SimpleStringEncoder<>("UTF-8"))
                .withOutputFileConfig(outputFileConfig)
                .withBucketAssigner(bucketAssigner)
                .withRollingPolicy(rollingPolicy).build();
        dataGen.sinkTo(fileSink);
        env.execute();
    }
}
